{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 相似文章推荐-Word2Vec+Tfidf+LSH"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 设置 python 和 spark路径"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init(spark_home='/usr/local/spark/',python_path='/home/master/LoadData/venv/bin/python')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导入hbase数据并进行分词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "SLF4J: Class path contains multiple SLF4J bindings.\n",
      "SLF4J: Found binding in [jar:file:/usr/local/spark/jars/slf4j-log4j12-1.7.30.jar!/org/slf4j/impl/StaticLoggerBinder.class]\n",
      "SLF4J: Found binding in [jar:file:/usr/local/hadoop/share/hadoop/common/lib/slf4j-log4j12-1.7.25.jar!/org/slf4j/impl/StaticLoggerBinder.class]\n",
      "SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.\n",
      "SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]\n",
      "2023-08-04 10:36:16,814 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "2023-08-04 10:36:18,505 WARN util.Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.\n",
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 0.552 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DataFrame[id: string, title: array<string>]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 0:>                                                          (0 + 1) / 1]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+--------------------------------+\n",
      "|                 id|                           title|\n",
      "+-------------------+--------------------------------+\n",
      "|4784374018419458049|    [穗, 港澳, 青少, 青少年, ...|\n",
      "|4784374018432040961|    [壮观, 新疆, 兵团, 麦收, ...|\n",
      "|4784374018444623873|    [浙江, 浙江省, 首个, 市, ...|\n",
      "|4784374018461401089|    [助力, 力特, 特色, 产品, ...|\n",
      "|4784374018482372609|    [中国, 京津, 京津冀, 晋, ...|\n",
      "|4784374018499149825|    [中电, 中电联, 电联, 月, ...|\n",
      "|4784374018511732737|    [横琴, 粤, 澳, 深度, 合作...|\n",
      "|4784374018524315649|    [省份, 专家, 共议, 打造, ...|\n",
      "|4784374018536898561|    [广东, 名, 青年, 志愿, 志...|\n",
      "|4784374018545287169|    [杭, 温, 铁路, 开始, 全线...|\n",
      "|4784374018591424513|      [种, 下, 这, 片, 蓝色, ...|\n",
      "|4784374018604007425|    [第, 届, 青洽会, 开幕, 绿...|\n",
      "|4784374018616590337|     [网, 传, 宁波, 一, 教师,...|\n",
      "|4784374018629173249|    [国家, 发改委, 将, 推出, ...|\n",
      "|4784374018637561857|    [杭州, 启动, 走遍, 杭州, ...|\n",
      "|4784374018645950465|    [福建, 泉州, 探索, 世界, ...|\n",
      "|4784374018658533377|    [截至, 月底, 累计, 家, 专...|\n",
      "|4784374018675310593|   [国家, 发改委, 两份, 促进,...|\n",
      "|4784374018687893505|   [餐饮, 餐饮店, 饮店, 下单,...|\n",
      "|4784374018696282113|  [中国, 绿色, 绿色生态, 生态...|\n",
      "|4784374018704670721|  [广西, 广西壮族, 壮族, 香火...|\n",
      "|4784374018713059329|   [上半, 上半年, 半年, 中国,...|\n",
      "|4784374018725642241|      [公益, 助, 农, 好, 物, ...|\n",
      "|4784374018738225153|   [杭州, 上半, 上半年, 半年,...|\n",
      "|4784374018750808065|    [天津, 打造, 海河, 河东, ...|\n",
      "|4784374018759196673|  [聚焦, 焦黄, 黄河, 黄河流域...|\n",
      "|4784374018767585281|    [渝, 贵, 铁路, 暑运, 期间...|\n",
      "|4784374018780168193|    [福建, 泉州, 探索, 世界, ...|\n",
      "|4784374018788556801|  [新疆, 第十, 第十四, 第十四...|\n",
      "|4784374018801139713|    [中国, 水稻, 育种, 获, 重...|\n",
      "|4784374018809528321|     [新型, 桥隧, 清, 筛, 车,...|\n",
      "|4784374018817916929|    [台, 研究, 机构, 大幅, 下...|\n",
      "|4784374018830499841|     [王, 贻, 芳, 院士, 理论,...|\n",
      "|4784374018843082753|  [中国, 中国科大, 科大, 已有...|\n",
      "|4784374018851471361|    [国家, 防总, 针对, 京津, ...|\n",
      "|4784374018864054273|  [国家, 国家开发银行, 开发, ...|\n",
      "|4784374018876637185|    [涉案, 逾, 亿元, 中国, 中...|\n",
      "|4784374018889220097|    [美国, 国强, 强力, 球, 彩...|\n",
      "|4784374018897608705|  [海南, 海南省, 海南省政府, ...|\n",
      "|4784374018910191617|    [年, 澳新, 女足, 世界, 世...|\n",
      "|4784374018922774529|    [违法, 违规, 收集, 使用, ...|\n",
      "|4784374018935357441|    [浙江, 位, 民间, 手艺, 手...|\n",
      "|4784374018943746049|      [乡, 约, 福建, 以, 侨, ...|\n",
      "|4784374018952134657|    [外, 媒, 阿根, 阿根廷, 根...|\n",
      "|4784374018964717569|    [中国, 国校, 校园, 健康, ...|\n",
      "|4784374018973106177|    [北京, 国际, 安全, 应急, ...|\n",
      "|4784374018985689089|    [足协, 连, 开, 四张, 罚单...|\n",
      "|4784374018994077697|    [乙脑, 进入, 入流, 流行, ...|\n",
      "|4784374019002466305|    [多元, 文化, 化探, 探究, ...|\n",
      "|4784374019010854913|   [工业, 互联, 互联网, 联网,...|\n",
      "|4784374019019243521|    [全国, 国特, 奥, 日, 享受...|\n",
      "|4784374019027632129|    [江苏, 徐州, 悬赏, 近, 万...|\n",
      "|4784374019040215041|    [中央, 重磅, 文件, 发布, ...|\n",
      "|4784374019048603649|   [四川, 成都, 东华, 东华门,...|\n",
      "|4784374019056992257|    [报告, 称, 大湾, 湾区, 年...|\n",
      "|4784374019065380865|   [长沙, 宠物, 宠物店, 推出,...|\n",
      "|4784374019073769473|   [今年, 上半, 上半年, 半年,...|\n",
      "|4784374019082158081|     [余, 伟, 文, 香港, 金融,...|\n",
      "|4784374019094740993|    [中国, 哈密, 甜蜜, 之, 旅...|\n",
      "|4784374019103129601|   [高质, 高质量, 质量, 发展,...|\n",
      "|4784374019111518209|    [作家, 家山, 山峰, 乡, 恋...|\n",
      "|4784374019119906817|    [女足, 世界, 世界杯, 年, ...|\n",
      "|4784374019132489729|     [杭州, 西湖, 第, 窝, 小,...|\n",
      "|4784374019140878337|    [国际, 网络, 达人, 在, 新...|\n",
      "|4784374019149266945|    [穗, 港澳, 青少, 青少年, ...|\n",
      "|4784374019161849857|    [壮观, 新疆, 兵团, 麦收, ...|\n",
      "|4784374019170238465|    [浙江, 浙江省, 首个, 市, ...|\n",
      "|4784374019178627073|    [助力, 力特, 特色, 产品, ...|\n",
      "|4784374019191209985|    [中国, 京津, 京津冀, 晋, ...|\n",
      "|4784374019199598593|    [中电, 中电联, 电联, 月, ...|\n",
      "|4784374019207987201|    [横琴, 粤, 澳, 深度, 合作...|\n",
      "|4784374019220570113|    [省份, 专家, 共议, 打造, ...|\n",
      "|4784374019233153025|    [广东, 名, 青年, 志愿, 志...|\n",
      "|4784374019241541633|    [杭, 温, 铁路, 开始, 全线...|\n",
      "|4784374019254124545|      [种, 下, 这, 片, 蓝色, ...|\n",
      "|4784374019262513153|    [第, 届, 青洽会, 开幕, 绿...|\n",
      "|4784374019275096065|     [网, 传, 宁波, 一, 教师,...|\n",
      "|4784374019287678977|    [国家, 发改委, 将, 推出, ...|\n",
      "|4784374019296067585|    [杭州, 启动, 走遍, 杭州, ...|\n",
      "|4784374019304456193|    [福建, 泉州, 探索, 世界, ...|\n",
      "|4784374019317039105|    [截至, 月底, 累计, 家, 专...|\n",
      "|4784374019325427713|   [上半, 上半年, 半年, 福建,...|\n",
      "|4784374019333816321|     [港, 大研, 研究, 指, 型,...|\n",
      "|4784374019342204929|  [台湾, 台湾大学, 大学, 大学...|\n",
      "|4784374019354787841|   [李小龙, 小龙, 逝世, 五十,...|\n",
      "|4784374019363176449|    [山西, 西面, 面向, 全球, ...|\n",
      "|4784374019375759361|    [江苏, 举办, 养老, 护理, ...|\n",
      "|4784374019384147969|    [山东, 厚, 植, 文化, 沃土...|\n",
      "|4784374019396730881|    [中国, 云南, 南自, 自由, ...|\n",
      "|4784374019405119489|    [中国, 寻根, 之, 旅, 夏令...|\n",
      "|4784374019413508097|    [广西, 发, 力, 修复, 漓江...|\n",
      "|4784374019426091009|    [湖南, 望城, 启动, 服务, ...|\n",
      "|4784374019438673921|  [中美, 应对, 气候, 气候变化...|\n",
      "|4784374019447062529|    [挺进, 地下, 万米, 这, 一...|\n",
      "|4784374019459645441|   [东南, 东南亚, 南亚, 孔子,...|\n",
      "|4784374019472228353|   [神舟, 十六, 十六号, 六号,...|\n",
      "|4784374019480616961|   [神舟, 十六, 十六号, 六号,...|\n",
      "|4784374019489005569|  [推进, 人与自然, 自然, 和谐...|\n",
      "|4784374019501588481|   [叙利亚, 利亚, 古代, 文物,...|\n",
      "|4784374019509977089|    [成都, 大运, 纪事, 成都, ...|\n",
      "|4784374019522560001|    [云南, 孟, 连成, 成为, 国...|\n",
      "|4784374045376249857|   [杭州, 亚运, 亚运会, 会首,...|\n",
      "|4784374045388832769|   [香港, 地方, 地方志, 方志,...|\n",
      "|4784374045397221377|    [李, 丰, 当选, 选为, 广东...|\n",
      "|4784374045409804289|    [北京, 延庆, 推动, 京, 张...|\n",
      "|4784374045422387201|    [信用, 赋, 能, 高质, 高质...|\n",
      "|4784374045434970113|    [江西, 到, 年底, 实现, 市...|\n",
      "|4784374045447553025|    [新朋, 故友, 聚, 此时, 台...|\n",
      "|4784374045460135937|    [实地, 探访, 四川, 成都, ...|\n",
      "|4784374045472718849|   [西安, 成立, 调查, 调查组,...|\n",
      "|4784374045481107457|    [赴, 广袤, 西部, 砺, 青春...|\n",
      "|4784374045489496065|  [中国, 中国文联, 国文, 文联...|\n",
      "|4784374045502078977|   [浙江, 江上, 上半, 上半年,...|\n",
      "|4784374045514661889|    [上海, 徐汇, 添置, 公共, ...|\n",
      "|4784374045523050497|    [聚焦, 人类, 生物, 分子, ...|\n",
      "|4784374045531439105|   [日本, 上半, 上半年, 半年,...|\n",
      "|4784374045544022017|    [大熊, 大熊猫, 熊猫, 卖, ...|\n",
      "|4784374045556604929|    [中国, 乡村, 发展, 基金, ...|\n",
      "|4784374045577576449|    [两岸, 媒体, 人, 访, 甘肃...|\n",
      "|4784374045590159361|   [第七, 第七届, 七届, 海峡,...|\n",
      "|4784374045602742273|    [粤港, 合作, 融合, 合出, ...|\n",
      "|4784374045611130881|    [链接, 全球, 球市, 市场, ...|\n",
      "|4784374045627908097|    [山西, 西兴, 兴县, 打造, ...|\n",
      "|4784374045636296705|    [江苏, 公布, 一批, 整治, ...|\n",
      "|4784374045648879617|  [中国, 中国航空学会, 国航, ...|\n",
      "|4784374045661462529|    [共建, 上海, 工业, 博物, ...|\n",
      "|4784374045674045441|    [北京, 延庆, 打造, 长城, ...|\n",
      "|4784374045682434049|     [上海, 黄浦, 星, 侨, 荟,...|\n",
      "|4784374045695016961|    [国航, 开通, 成都, 吉隆, ...|\n",
      "|4784374045703405569|    [历时, 六年, 精心, 打磨, ...|\n",
      "|4784374045711794177|    [安徽, 合肥, 感, 旗袍, 之...|\n",
      "|4784374045724377089|[中华, 中华全国, 中华全国总工...|\n",
      "|4784374045736960001|    [国际, 最新, 研究, 韦, 布...|\n",
      "|4784374045745348609|   [神舟, 十六, 十六号, 六号,...|\n",
      "|4784374045757931521|  [中国, 国企, 企业, 企业信用...|\n",
      "|4784374045766320129|    [北京, 台胞, 走进, 腾讯, ...|\n",
      "|4784374045774708737|     [中国, 驻, 美, 大使, 谢,...|\n",
      "|4784374045787291649|    [谢, 锋, 谈, 中美, 中美关...|\n",
      "|4784374045795680257|    [桂, 港澳, 青年, 青年组, ...|\n",
      "|4784374045808263169|    [中国, 将, 加快, 制定, 政...|\n",
      "|4784374045816651777|   [女足, 世界, 世界杯, 综合,...|\n",
      "|4784374045825040385|    [学者, 万里, 为, 邻, 钢铁...|\n",
      "|4784374045837623297|    [中外, 外学, 学者, 一带, ...|\n",
      "|4784374045846011905|   [澳新, 女足, 世界, 世界杯,...|\n",
      "|4784374045854400513|     [萌, 化, 了, 南京, 京海,...|\n",
      "|4784374045862789121|   [杭州, 亚运, 亚运会, 会首,...|\n",
      "|4784374045875372033|    [遇见, 福建, 航拍, 迷人, ...|\n",
      "|4784374045883760641|   [台湾, 台湾网, 网络, 达人,...|\n",
      "|4784374045892149249|    [江河, 交汇, 汇水, 水域, ...|\n",
      "|4784374045904732161|  [外资, 外资企业, 企业, 地方...|\n",
      "+-------------------+--------------------------------+\n",
      "only showing top 150 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "import happybase\n",
    "import jieba\n",
    "from txdpy import get_chinese\n",
    "\n",
    "spark = SparkSession.builder.appName('SparkHBaseRDD').master('local[*]').getOrCreate()\n",
    "sc=spark.sparkContext\n",
    "\n",
    "connection=happybase.Connection('master')\n",
    "table=connection.table('news')\n",
    "g=table.scan()\n",
    "data_list = []\n",
    "for k, d in g:\n",
    "    new_d = {}\n",
    "    new_d[\"id\"] = str(k, 'utf-8')\n",
    "    # new_d[\"year\"] = str(d[b\"info:year\"], 'utf-8')\n",
    "    # for key, value in d.items():\n",
    "    #     new_d[f\"{str(key, 'utf-8')}\"] = str(value, 'utf-8')\n",
    "    new_title = [i for i in get_chinese(str(d[b\"info:title\"], 'utf-8'))]\n",
    "    new_d[\"title\"] = [i for i in jieba.cut(str(''.join(new_title)), cut_all=True)]\n",
    "    \n",
    "    data_list.append(new_d)\n",
    "\n",
    "df = spark.createDataFrame(data_list)\n",
    "print(df)\n",
    "df.show(150)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## word2vec 训练分词数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-08-04 10:36:43,246 WARN netlib.BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS\n",
      "2023-08-04 10:36:43,260 WARN netlib.BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS\n",
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.feature import Word2Vec\n",
    "\n",
    "w2v_model = Word2Vec(vectorSize=64, inputCol='title', outputCol='vector', minCount=3)\n",
    "model = w2v_model.fit(df)\n",
    "model.write().overwrite().save(\"models/word2vec_model/python.word2vec\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+--------------------+\n",
      "|word|              vector|\n",
      "+----+--------------------+\n",
      "|伙伴|[-0.0163482651114...|\n",
      "|人物|[-0.0116263534873...|\n",
      "|晋级|[0.00441837357357...|\n",
      "|确保|[-0.0040366929024...|\n",
      "|接种|[-0.0015587668167...|\n",
      "|  色|[-0.0065059727057...|\n",
      "|冠军|[-0.0144163947552...|\n",
      "|  被|[-0.0314384922385...|\n",
      "|  较|[0.00984632782638...|\n",
      "|首枚|[-0.0090482272207...|\n",
      "|球队|[3.57587297912687...|\n",
      "|约合|[-0.0041686976328...|\n",
      "|  玟|[-3.0482787406072...|\n",
      "|  传|[0.00331560312770...|\n",
      "|一天|[0.01256527658551...|\n",
      "|长沙|[-0.0082209175452...|\n",
      "|我们|[0.02708181925117...|\n",
      "|共生|[0.02342626824975...|\n",
      "|步枪|[-3.5825232043862...|\n",
      "|贵宾|[0.02632013708353...|\n",
      "+----+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.feature import Word2VecModel\n",
    "\n",
    "w2v_model_e = Word2VecModel.load(\"models/word2vec_model/python.word2vec\")\n",
    "vectors = w2v_model_e.getVectors()\n",
    "vectors.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import CountVectorizer\n",
    "\n",
    "cv = CountVectorizer(inputCol=\"title\", outputCol=\"countFeatures\", vocabSize=200 * 10000, minDF=1.0)\n",
    "cv_model = cv.fit(df)\n",
    "cv_model.write().overwrite().save(\"models/CV.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+------------------------------+--------------------+\n",
      "|                 id|                         title|       countFeatures|\n",
      "+-------------------+------------------------------+--------------------+\n",
      "|4784374018419458049|  [穗, 港澳, 青少, 青少年, ...|(16692,[34,84,104...|\n",
      "|4784374018432040961|  [壮观, 新疆, 兵团, 麦收, ...|(16692,[35,145,55...|\n",
      "|4784374018444623873|  [浙江, 浙江省, 首个, 市, ...|(16692,[28,33,45,...|\n",
      "|4784374018461401089|  [助力, 力特, 特色, 产品, ...|(16692,[37,56,80,...|\n",
      "|4784374018482372609|  [中国, 京津, 京津冀, 晋, ...|(16692,[0,4,112,1...|\n",
      "|4784374018499149825|  [中电, 中电联, 电联, 月, ...|(16692,[0,43,68,1...|\n",
      "|4784374018511732737|  [横琴, 粤, 澳, 深度, 合作...|(16692,[28,111,57...|\n",
      "|4784374018524315649|  [省份, 专家, 共议, 打造, ...|(16692,[75,101,13...|\n",
      "|4784374018536898561|  [广东, 名, 青年, 志愿, 志...|(16692,[54,110,12...|\n",
      "|4784374018545287169|  [杭, 温, 铁路, 开始, 全线...|(16692,[4,126,169...|\n",
      "|4784374018591424513|    [种, 下, 这, 片, 蓝色, ...|(16692,[82,228,24...|\n",
      "|4784374018604007425|  [第, 届, 青洽会, 开幕, 绿...|(16692,[37,170,17...|\n",
      "|4784374018616590337|   [网, 传, 宁波, 一, 教师,...|(16692,[25,60,66,...|\n",
      "|4784374018629173249|  [国家, 发改委, 将, 推出, ...|(16692,[8,11,80,3...|\n",
      "|4784374018637561857|  [杭州, 启动, 走遍, 杭州, ...|(16692,[1,34,163,...|\n",
      "|4784374018645950465|  [福建, 泉州, 探索, 世界, ...|(16692,[17,32,162...|\n",
      "|4784374018658533377|  [截至, 月底, 累计, 家, 专...|(16692,[0,2,12,36...|\n",
      "|4784374018675310593| [国家, 发改委, 两份, 促进,...|(16692,[4,8,11,19...|\n",
      "|4784374018687893505| [餐饮, 餐饮店, 饮店, 下单,...|(16692,[56,60,327...|\n",
      "|4784374018696282113|[中国, 绿色, 绿色生态, 生态...|(16692,[0,1,23,11...|\n",
      "+-------------------+------------------------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.feature import CountVectorizerModel\n",
    "\n",
    "cv_model = CountVectorizerModel.load(\"models/CV.model\")\n",
    "cv_result = cv_model.transform(df)\n",
    "cv_result.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.feature import IDF\n",
    "\n",
    "idf = IDF(inputCol=\"countFeatures\", outputCol=\"idfFeatures\")\n",
    "idf_model = idf.fit(cv_result)\n",
    "idf_model.write().overwrite().save(\"models/IDF.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+------------------------------+--------------------+--------------------+\n",
      "|                 id|                         title|       countFeatures|         idfFeatures|\n",
      "+-------------------+------------------------------+--------------------+--------------------+\n",
      "|4784374018419458049|  [穗, 港澳, 青少, 青少年, ...|(16692,[34,84,104...|(16692,[34,84,104...|\n",
      "|4784374018432040961|  [壮观, 新疆, 兵团, 麦收, ...|(16692,[35,145,55...|(16692,[35,145,55...|\n",
      "|4784374018444623873|  [浙江, 浙江省, 首个, 市, ...|(16692,[28,33,45,...|(16692,[28,33,45,...|\n",
      "|4784374018461401089|  [助力, 力特, 特色, 产品, ...|(16692,[37,56,80,...|(16692,[37,56,80,...|\n",
      "|4784374018482372609|  [中国, 京津, 京津冀, 晋, ...|(16692,[0,4,112,1...|(16692,[0,4,112,1...|\n",
      "|4784374018499149825|  [中电, 中电联, 电联, 月, ...|(16692,[0,43,68,1...|(16692,[0,43,68,1...|\n",
      "|4784374018511732737|  [横琴, 粤, 澳, 深度, 合作...|(16692,[28,111,57...|(16692,[28,111,57...|\n",
      "|4784374018524315649|  [省份, 专家, 共议, 打造, ...|(16692,[75,101,13...|(16692,[75,101,13...|\n",
      "|4784374018536898561|  [广东, 名, 青年, 志愿, 志...|(16692,[54,110,12...|(16692,[54,110,12...|\n",
      "|4784374018545287169|  [杭, 温, 铁路, 开始, 全线...|(16692,[4,126,169...|(16692,[4,126,169...|\n",
      "|4784374018591424513|    [种, 下, 这, 片, 蓝色, ...|(16692,[82,228,24...|(16692,[82,228,24...|\n",
      "|4784374018604007425|  [第, 届, 青洽会, 开幕, 绿...|(16692,[37,170,17...|(16692,[37,170,17...|\n",
      "|4784374018616590337|   [网, 传, 宁波, 一, 教师,...|(16692,[25,60,66,...|(16692,[25,60,66,...|\n",
      "|4784374018629173249|  [国家, 发改委, 将, 推出, ...|(16692,[8,11,80,3...|(16692,[8,11,80,3...|\n",
      "|4784374018637561857|  [杭州, 启动, 走遍, 杭州, ...|(16692,[1,34,163,...|(16692,[1,34,163,...|\n",
      "|4784374018645950465|  [福建, 泉州, 探索, 世界, ...|(16692,[17,32,162...|(16692,[17,32,162...|\n",
      "|4784374018658533377|  [截至, 月底, 累计, 家, 专...|(16692,[0,2,12,36...|(16692,[0,2,12,36...|\n",
      "|4784374018675310593| [国家, 发改委, 两份, 促进,...|(16692,[4,8,11,19...|(16692,[4,8,11,19...|\n",
      "|4784374018687893505| [餐饮, 餐饮店, 饮店, 下单,...|(16692,[56,60,327...|(16692,[56,60,327...|\n",
      "|4784374018696282113|[中国, 绿色, 绿色生态, 生态...|(16692,[0,1,23,11...|(16692,[0,1,23,11...|\n",
      "+-------------------+------------------------------+--------------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.feature import IDFModel\n",
    "\n",
    "idf_model = IDFModel.load(\"models/IDF.model\")\n",
    "tfidf_result = idf_model.transform(cv_result)\n",
    "tfidf_result.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sort_by_tfidf(partition):\n",
    "    TOPK = 20\n",
    "    for row in partition:\n",
    "        _dict = list(zip(row.idfFeatures.indices, row.idfFeatures.values))\n",
    "        _dict = sorted(_dict, key=lambda x: x[1], reverse=True)\n",
    "        result = _dict[:TOPK]\n",
    "        for word_index, tfidf in result:\n",
    "            yield row.id, int(word_index), round(float(tfidf), 4)\n",
    "\n",
    "\n",
    "keywords_by_tfidf = tfidf_result.rdd.mapPartitions(sort_by_tfidf).toDF([\"id\", \"index\", \"weights\"])\n",
    "\n",
    "keywords_by_tfidf.show()\n",
    "\n",
    "keywords_list_with_idf = list(zip(cv_model.vocabulary, idf_model.idf.toArray()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def append_index(data):\n",
    "    for index in range(len(data)):\n",
    "        data[index] = list(data[index])  # 将元组转为list\n",
    "        data[index].append(index)  # 加入索引\n",
    "        data[index][1] = float(data[index][1])\n",
    " \n",
    "append_index(keywords_list_with_idf)\n",
    "keywords_list_with_idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = spark.sparkContext\n",
    "rdd = sc.parallelize(keywords_list_with_idf)  # 创建rdd\n",
    "idf_keywords = rdd.toDF([\"keywords\", \"idf\", \"index\"])\n",
    "idf_keywords.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "keywords_result = keywords_by_tfidf.join(idf_keywords, idf_keywords.index == keywords_by_tfidf.index).select(\n",
    "    [\"id\", \"keywords\", \"weights\"])\n",
    "keywords_result.rdd.toDF([\"id\", \"keywords\", \"weights\"]).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "keywords_vector = keywords_result.join(vectors, vectors.word == keywords_result.keywords, 'inner')\n",
    "keywords_vector.rdd.toDF([\"id\", \"keywords\", \"weights\"]).show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_vector(row):\n",
    "    return row.id, row.keywords, row.weights * row.vector\n",
    " \n",
    "article_keyword_vectors = keywords_vector.rdd.map(compute_vector).toDF([\"id\", \"keywords\", \"weightingVector\"])\n",
    " \n",
    "# 利用 collect_set() 方法，将一篇文章内所有关键词的词向量合并为一个列表\n",
    "article_keyword_vectors.registerTempTable('temptable')\n",
    "article_keyword_vectors = spark.sql(\"select id, collect_set(weightingVector) vectors from temptable group by id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_avg_vectors(row):\n",
    "    x = 0\n",
    "    for i in row.vectors:\n",
    "        x += i\n",
    "    # 求平均值\n",
    "    return row.id, x / len(row.vectors)\n",
    " \n",
    "article_vector = article_keyword_vectors.rdd.map(compute_avg_vectors).toDF(['id', 'articlevector'])\n",
    "print(\"文章最终vector\",article_vector.take(10))\n",
    "article_vector.rdd.toDF(['id', 'articlevector']).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LSH\n",
    "from pyspark.ml.feature import BucketedRandomProjectionLSH, MinHashLSH\n",
    " \n",
    "train = article_vector.select(['id', 'articlevector'])\n",
    " \n",
    "# 1.BucketedRandomProjectionLSH\n",
    "brp = BucketedRandomProjectionLSH(inputCol='articlevector', outputCol='hashes', numHashTables=4.0, bucketLength=10.0)\n",
    "model = brp.fit(train)\n",
    " \n",
    "similar = model.approxSimilarityJoin(train, train, 2.0, distCol='EuclideanDistance')\n",
    "similar.show()\n",
    "\n",
    "# 2.MinHashLSH\n",
    "# brp = MinHashLSH(inputCol='articlevector', outputCol='hashes', numHashTables=4.0)\n",
    "# model = brp.fit(train)\n",
    " \n",
    "# 获取所有相似对\n",
    "# similar = model.approxSimilarityJoin(train, train, 2.0, distCol='EuclideanDistance')\n",
    "# similar.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.write().overwrite().save(\"models/LSH.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import BucketedRandomProjectionLSHModel\n",
    "\n",
    "LSHmodel = BucketedRandomProjectionLSHModel.load(\"models/LSH.model\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
